import asyncio
import aiohttp
import json
import pymongo

def process_and_insert(collection, data, page):
    try:
        data_to_insert = []
        for zq in data['data']:
            data_to_insert.append({
                "债券名称": zq['ZQ_NAME'],
                "债券编码": zq['ZQ_CODE'],
                "发行地区": zq['AD_NAME'],
                "发行时间": zq['ZQ_FXTIME'],
                "发行期限": zq['ZQQX_NAME'],
                "发行规模": zq['FX_AMT'],
                "利率": zq['LL'],
                "付息方式": zq['FXFS'],
                "起息日": zq['QX_DATE']
            })
            print(f"正在爬取第{page}页债券{zq['ZQ_CODE']}......")
        if data_to_insert:
            collection.insert_many(data_to_insert)
    except pymongo.errors.PyMongoError as e:
        print(f"插入第 {page} 页数据到数据库失败: {e}")


# 异步获取数据并处理
async def fetch_and_insert(session, page, client):
    db_name = '中国地方政府债券协程版'
    db = client[db_name]
    collection = db['data']
    url = f"https://www.governbond.org.cn:4443/api/loadBondData.action?timeStamp=1744278770234&dataType=ZQFXLISTBYAD&adList=&adCode=87&zqlx=&year=&fxfs=&qxr=&fxqx=&zqCode=&zqName=&page={page}&pageSize=10"
    headers = {
        "user-agent": "Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Mobile Safari/537.36"
    }
    try:
        async with session.get(url, headers=headers) as response:
            response.raise_for_status()
            html = await response.text()
            data = json.loads(html)
            process_and_insert(collection, data, page)
    except aiohttp.ClientError as e:
        print(f"请求第 {page} 页失败: {e}")
    except json.JSONDecodeError as e:
        print(f"解析第 {page} 页的JSON数据失败: {e}")
    # 添加请求间隔
    await asyncio.sleep(0.1)


async def main():
    client = pymongo.MongoClient('mongodb://localhost:27017/')
    db_name = '中国地方政府债券协程版'
    client.drop_database(db_name)

    async with aiohttp.ClientSession() as session:
        tasks = []
        for page in range(1, 1642):
            task = asyncio.create_task(fetch_and_insert(session, page, client))
            tasks.append(task)
        await asyncio.gather(*tasks)

    client.close()


if __name__ == "__main__":
    asyncio.run(main())
